package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"log"
	"net/http"
)

func Fetch(url string) ([] byte, error) {

	resp, err := http.Get(url)

	if err != nil {
		return nil, err
	}
	//defer 定义的语句，会在函数退出前调用 类似 finally
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	htmlEncoding := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, htmlEncoding.NewDecoder())

	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(r *bufio.Reader) encoding.Encoding {

	bytes, err := r.Peek(1024)

	if err != nil {
		log.Printf("Fetcher error: %v",err)
		return unicode.UTF8
	}

	htmlEncoding, _, _ := charset.DetermineEncoding(bytes, "")

	return htmlEncoding
}
